require 'stringio'
require 'oga'
require 'proxy_pump/const'
module ProxyPump
  class Crawler

    extend  Forwardable

    def_delegators :client, :http, :user_agents

    attr_reader :client, :retry_times

    def initialize(client, options = {})
      @client      = client
      @options     = options
      @retry_times = options.fetch(:retry_times, Const::DEFAULT_RETRY_TIMES)
    end

    def crawl(source, options = {}, &block)
      max_retry_times = options.fetch(:retry_times, retry_times)
      begin
        url = source.url
        res = http.get url
        if res.status.success?
          collect_proxies(source, res, &block)
        elsif res.status.redirect?
          # If redirect let's try.
          # I think recursion is unnecessary.
          res = http.cookies(res.cookies).get(url)
          if res.status.success?
            collect_proxies(source, res, &block)
          end
        end
      rescue => e
        ProxyPump.logger.debug "Crawl site #{url} failed: #{max_retry_times} retry times left. (#{e.message}) (#{e.class})"
        max_retry_times -= 1
        if max_retry_times > 0
          sleep 0.1
          retry
        else
          raise e
        end
      end
    end

    private

      def collect_proxies(source, response, &block)
        converted_html = smart_convert_encoding(response.to_s)
        document       = Oga.parse_html StringIO.new(converted_html)
        proxies = document.public_send(source.entry_selector_type, source.entry_selector).map do |entry|
          host = entry.public_send(source.host_selector_type, source.host_selector).map(&:text).map(&:strip).join
          port = entry.public_send(source.port_selector_type, source.port_selector).map(&:text).map(&:strip).join.to_i
          anon = entry.public_send(source.anon_selector_type, source.anon_selector).map(&:text).map(&:strip).join
          anon = anon.scan(/透明|普匿|匿名|高匿|普通/).first || anon rescue anon
          protocal = entry.public_send(source.protocal_selector_type, source.protocal_selector).map(&:text).map(&:strip).join
          { host: host, port: port, anon: anon, protocal: protocal, source_name: source.name.to_s, source_url: source.url }
        end
        if block_given?
          yield proxies
        else
          proxies
        end
      end

      def smart_convert_encoding(str)
        case str.encoding
        when Encoding::ASCII_8BIT # fake encoding
          str.force_encoding(Encoding::GB2312).encode!(Encoding::UTF_8, Encoding::GB2312, {:invalid => :replace})
        when Encoding::GB2312
          str.encode!(Encoding::UTF_8, Encoding::GB2312, {:invalid => :replace})
        else
          str
        end
      end

  end # Crawler
end # ProxyPump
